<?xml version='1.0' encoding="utf-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html lang="en" xml:lang="en" xmlns="http://www.w3.org/1999/xhtml">
  <head>
    <link href="cpip.css" rel="stylesheet" type="text/css" />
    <title>File: /Users/paulross/dev/linux/linux-3.13/arch/x86/include/asm/bitops.h</title>
  </head>
  <body>
    <h1>File: /Users/paulross/dev/linux/linux-3.13/arch/x86/include/asm/bitops.h</h1>
    <p>Green shading in the line number column
means the source is part of the translation unit, red means it is conditionally excluded.
Highlighted line numbers link to the translation unit page. Highlighted macros link to
the macro page.</p>
    <pre><a name="1" /><span class="Maybe">       1:</span> <span class="f">#</span><span class="n">ifndef</span> <a href="cpu.c_macros_ref.html#_X0FTTV9YODZfQklUT1BTX0hfMA__"><span class="b">_ASM_X86_BITOPS_H</span></a>
<a name="2" /><span class="Maybe">       2:</span> <span class="f">#</span><span class="n">define</span> <a href="cpu.c_macros_ref.html#_X0FTTV9YODZfQklUT1BTX0hfMA__"><span class="b">_ASM_X86_BITOPS_H</span></a>
<a name="3" /><span class="Maybe">       3:</span> 
<a name="4" /><span class="Maybe">       4:</span> <span class="k">/*</span>
<a name="5" /><span class="Maybe">       5:</span> <span class="k"> * Copyright 1992, Linus Torvalds.</span>
<a name="6" /><span class="Maybe">       6:</span> <span class="k"> *</span>
<a name="7" /><span class="Maybe">       7:</span> <span class="k"> * Note: inlines with more than a single statement should be marked</span>
<a name="8" /><span class="Maybe">       8:</span> <span class="k"> * __always_inline to avoid problems with older gcc&apos;s inlining heuristics.</span>
<a name="9" /><span class="Maybe">       9:</span> <span class="k"> */</span>
<a name="10" /><span class="Maybe">      10:</span> 
<a name="11" /><span class="False">      11:</span> <span class="f">#</span><span class="n">ifndef</span> <a href="cpu.c_macros_ref.html#_X0xJTlVYX0JJVE9QU19IXzA_"><span class="b">_LINUX_BITOPS_H</span></a>
<a name="12" /><span class="False">      12:</span> <span class="f">#</span><span class="n">error</span> <span class="b">only</span> <span class="f">&lt;</span><span class="b">linux</span><span class="f">/</span><span class="b">bitops</span><span class="f">.</span><span class="b">h</span><span class="f">&gt;</span> <span class="b">can</span> <span class="b">be</span> <span class="b">included</span> <span class="b">directly</span>
<a name="13" /><span class="Maybe">      13:</span> <span class="f">#</span><span class="n">endif</span>
<a name="14" /><span class="Maybe">      14:</span> 
<a name="15" /><span class="Maybe">      15:</span> <span class="f">#</span><span class="n">include</span> <span class="f">&lt;</span><span class="b">linux</span><span class="f">/</span><span class="b">compiler</span><span class="f">.</span><span class="b">h</span><span class="f">&gt;</span>
<a name="16" /><span class="Maybe">      16:</span> <span class="f">#</span><span class="n">include</span> <span class="f">&lt;</span><span class="m">asm</span><span class="f">/</span><a href="cpu.c_macros_ref.html#_YWx0ZXJuYXRpdmVfMA__"><span class="b">alternative</span></a><span class="f">.</span><span class="b">h</span><span class="f">&gt;</span>
<a name="17" /><span class="Maybe">      17:</span> <span class="f">#</span><span class="n">include</span> <span class="f">&lt;</span><span class="m">asm</span><span class="f">/</span><span class="b">rmwcc</span><span class="f">.</span><span class="b">h</span><span class="f">&gt;</span>
<a name="18" /><span class="Maybe">      18:</span> 
<a name="19" /><span class="False">      19:</span> <span class="f">#</span><span class="n">if</span> <a href="cpu.c_macros_ref.html#_QklUU19QRVJfTE9OR18w"><span class="b">BITS_PER_LONG</span></a> <span class="f">==</span> <span class="c">32</span>
<a name="20" /><span class="False">      20:</span> <span class="f">#</span> <span class="n">define</span> <a href="cpu.c_macros_ref.html#_X0JJVE9QU19MT05HX1NISUZUXzA_"><span class="b">_BITOPS_LONG_SHIFT</span></a> <span class="c">5</span>
<a name="21" /><span class="Maybe">      21:</span> <span class="f">#</span><span class="n">elif</span> <a href="cpu.c_macros_ref.html#_QklUU19QRVJfTE9OR18w"><span class="b">BITS_PER_LONG</span></a> <span class="f">==</span> <span class="c">64</span>
<a name="22" /><span class="Maybe">      22:</span> <span class="f">#</span> <span class="n">define</span> <a href="cpu.c_macros_ref.html#_X0JJVE9QU19MT05HX1NISUZUXzA_"><span class="b">_BITOPS_LONG_SHIFT</span></a> <span class="c">6</span>
<a name="23" /><span class="False">      23:</span> <span class="f">#</span><span class="n">else</span>
<a name="24" /><span class="False">      24:</span> <span class="f">#</span> <span class="n">error</span> <span class="e">&quot;Unexpected BITS_PER_LONG&quot;</span>
<a name="25" /><span class="Maybe">      25:</span> <span class="f">#</span><span class="n">endif</span>
<a name="26" /><span class="Maybe">      26:</span> 
<a name="27" /><span class="Maybe">      27:</span> <span class="f">#</span><span class="n">define</span> <a href="cpu.c_macros_noref.html#_QklUXzY0XzA_"><span class="b">BIT_64</span></a><span class="f">(</span><span class="b">n</span><span class="f">)</span>            <span class="f">(</span><a href="cpu.c_macros_noref.html#_VTY0X0NfMA__"><span class="b">U64_C</span></a><span class="f">(</span><span class="c">1</span><span class="f">)</span> <span class="f">&lt;&lt;</span> <span class="f">(</span><span class="b">n</span><span class="f">)</span><span class="f">)</span>
<a name="28" /><span class="Maybe">      28:</span> 
<a name="29" /><span class="Maybe">      29:</span> <span class="k">/*</span>
<a name="30" /><span class="Maybe">      30:</span> <span class="k"> * These have to be done with inline assembly: that way the bit-setting</span>
<a name="31" /><span class="Maybe">      31:</span> <span class="k"> * is guaranteed to be atomic. All bit operations return 0 if the bit</span>
<a name="32" /><span class="Maybe">      32:</span> <span class="k"> * was cleared before the operation and != 0 if it was not.</span>
<a name="33" /><span class="Maybe">      33:</span> <span class="k"> *</span>
<a name="34" /><span class="Maybe">      34:</span> <span class="k"> * bit 0 is the LSB of addr; bit 32 is the LSB of (addr+1).</span>
<a name="35" /><span class="Maybe">      35:</span> <span class="k"> */</span>
<a name="36" /><span class="Maybe">      36:</span> 
<a name="37" /><span class="Maybe">      37:</span> <span class="f">#</span><span class="n">if</span> <a href="cpu.c_macros_ref.html#_X19HTlVDX19fMA__"><span class="b">__GNUC__</span></a> <span class="f">&lt;</span> <span class="c">4</span> <span class="f">||</span> <span class="f">(</span><a href="cpu.c_macros_ref.html#_X19HTlVDX19fMA__"><span class="b">__GNUC__</span></a> <span class="f">==</span> <span class="c">4</span> <span class="f">&amp;&amp;</span> <span class="b">__GNUC_MINOR__</span> <span class="f">&lt;</span> <span class="c">1</span><span class="f">)</span>
<a name="38" /><span class="Maybe">      38:</span> <span class="k">/* Technically wrong, but this avoids compilation errors on some gcc</span>
<a name="39" /><span class="Maybe">      39:</span> <span class="k">   versions. */</span>
<a name="40" /><span class="Maybe">      40:</span> <span class="f">#</span><span class="n">define</span> <a href="cpu.c_macros_ref.html#_QklUT1BfQUREUl8w"><span class="b">BITOP_ADDR</span></a><span class="f">(</span><span class="b">x</span><span class="f">)</span> <span class="e">&quot;=m&quot;</span> <span class="f">(</span><span class="f">*</span><span class="f">(</span><span class="m">volatile</span> <span class="m">long</span> <span class="f">*</span><span class="f">)</span> <span class="f">(</span><span class="b">x</span><span class="f">)</span><span class="f">)</span>
<a name="41" /><span class="False">      41:</span> <span class="f">#</span><span class="n">else</span>
<a name="42" /><span class="False">      42:</span> <span class="f">#</span><span class="n">define</span> <a href="cpu.c_macros_ref.html#_QklUT1BfQUREUl8w"><span class="b">BITOP_ADDR</span></a><span class="f">(</span><span class="b">x</span><span class="f">)</span> <span class="e">&quot;+m&quot;</span> <span class="f">(</span><span class="f">*</span><span class="f">(</span><span class="m">volatile</span> <span class="m">long</span> <span class="f">*</span><span class="f">)</span> <span class="f">(</span><span class="b">x</span><span class="f">)</span><span class="f">)</span>
<a name="43" /><span class="Maybe">      43:</span> <span class="f">#</span><span class="n">endif</span>
<a name="44" /><span class="Maybe">      44:</span> 
<a name="45" /><span class="Maybe">      45:</span> <span class="f">#</span><span class="n">define</span> <a href="cpu.c_macros_ref.html#_QUREUl8w"><span class="b">ADDR</span></a>                <a href="cpu.c_macros_ref.html#_QklUT1BfQUREUl8w"><span class="b">BITOP_ADDR</span></a><span class="f">(</span><span class="b">addr</span><span class="f">)</span>
<a name="46" /><span class="Maybe">      46:</span> 
<a name="47" /><span class="Maybe">      47:</span> <span class="k">/*</span>
<a name="48" /><span class="Maybe">      48:</span> <span class="k"> * We do the locked ops that don&apos;t return the old value as</span>
<a name="49" /><span class="Maybe">      49:</span> <span class="k"> * a mask operation on a byte.</span>
<a name="50" /><span class="Maybe">      50:</span> <span class="k"> */</span>
<a name="51" /><span class="Maybe">      51:</span> <span class="f">#</span><span class="n">define</span> <a href="cpu.c_macros_ref.html#_SVNfSU1NRURJQVRFXzA_"><span class="b">IS_IMMEDIATE</span></a><span class="f">(</span><span class="b">nr</span><span class="f">)</span>        <span class="f">(</span><span class="b">__builtin_constant_p</span><span class="f">(</span><span class="b">nr</span><span class="f">)</span><span class="f">)</span>
<a name="52" /><span class="Maybe">      52:</span> <span class="f">#</span><span class="n">define</span> <a href="cpu.c_macros_ref.html#_Q09OU1RfTUFTS19BRERSXzA_"><span class="b">CONST_MASK_ADDR</span></a><span class="f">(</span><span class="b">nr</span><span class="f">,</span> <span class="b">addr</span><span class="f">)</span>    <a href="cpu.c_macros_ref.html#_QklUT1BfQUREUl8w"><span class="b">BITOP_ADDR</span></a><span class="f">(</span><span class="f">(</span><span class="m">void</span> <span class="f">*</span><span class="f">)</span><span class="f">(</span><span class="b">addr</span><span class="f">)</span> <span class="f">+</span> <span class="f">(</span><span class="f">(</span><span class="b">nr</span><span class="f">)</span><span class="f">&gt;&gt;</span><span class="c">3</span><span class="f">)</span><span class="f">)</span>
<a name="53" /><span class="Maybe">      53:</span> <span class="f">#</span><span class="n">define</span> <a href="cpu.c_macros_ref.html#_Q09OU1RfTUFTS18w"><span class="b">CONST_MASK</span></a><span class="f">(</span><span class="b">nr</span><span class="f">)</span>            <span class="f">(</span><span class="c">1</span> <span class="f">&lt;&lt;</span> <span class="f">(</span><span class="f">(</span><span class="b">nr</span><span class="f">)</span> <span class="f">&amp;</span> <span class="c">7</span><span class="f">)</span><span class="f">)</span>
<a name="54" /><span class="Maybe">      54:</span> 
<a name="55" /><span class="Maybe">      55:</span> <span class="k">/**</span>
<a name="56" /><span class="Maybe">      56:</span> <span class="k"> * set_bit - Atomically set a bit in memory</span>
<a name="57" /><span class="Maybe">      57:</span> <span class="k"> * @nr: the bit to set</span>
<a name="58" /><span class="Maybe">      58:</span> <span class="k"> * @addr: the address to start counting from</span>
<a name="59" /><span class="Maybe">      59:</span> <span class="k"> *</span>
<a name="60" /><span class="Maybe">      60:</span> <span class="k"> * This function is atomic and may not be reordered.  See __set_bit()</span>
<a name="61" /><span class="Maybe">      61:</span> <span class="k"> * if you do not require the atomic guarantees.</span>
<a name="62" /><span class="Maybe">      62:</span> <span class="k"> *</span>
<a name="63" /><span class="Maybe">      63:</span> <span class="k"> * Note: there are no guarantees that this function will not be reordered</span>
<a name="64" /><span class="Maybe">      64:</span> <span class="k"> * on non x86 architectures, so if you are writing portable code,</span>
<a name="65" /><span class="Maybe">      65:</span> <span class="k"> * make sure not to rely on its reordering guarantees.</span>
<a name="66" /><span class="Maybe">      66:</span> <span class="k"> *</span>
<a name="67" /><span class="Maybe">      67:</span> <span class="k"> * Note that @nr may be almost arbitrarily large; this function is not</span>
<a name="68" /><span class="Maybe">      68:</span> <span class="k"> * restricted to acting on a single-word quantity.</span>
<a name="69" /><span class="Maybe">      69:</span> <span class="k"> */</span>
<a name="70" /><span class="Maybe">      70:</span> <span class="m">static</span> <a href="cpu.c_macros_ref.html#_X19hbHdheXNfaW5saW5lXzA_"><span class="b">__always_inline</span></a> <span class="m">void</span>
<a name="71" /><span class="Maybe">      71:</span> <span class="b">set_bit</span><span class="f">(</span><span class="m">long</span> <span class="b">nr</span><span class="f">,</span> <span class="m">volatile</span> <span class="m">unsigned</span> <span class="m">long</span> <span class="f">*</span><span class="b">addr</span><span class="f">)</span>
<a name="72" /><span class="Maybe">      72:</span> <span class="f">{</span>
<a name="73" /><span class="Maybe">      73:</span>     <span class="m">if</span> <span class="f">(</span><a href="cpu.c_macros_ref.html#_SVNfSU1NRURJQVRFXzA_"><span class="b">IS_IMMEDIATE</span></a><span class="f">(</span><span class="b">nr</span><span class="f">)</span><span class="f">)</span> <span class="f">{</span>
<a name="74" /><span class="Maybe">      74:</span>         <span class="m">asm</span> <span class="m">volatile</span><span class="f">(</span><a href="cpu.c_macros_ref.html#_TE9DS19QUkVGSVhfMA__"><span class="b">LOCK_PREFIX</span></a> <span class="e">&quot;orb %1,%0&quot;</span>
<a name="75" /><span class="Maybe">      75:</span>             <span class="f">:</span> <a href="cpu.c_macros_ref.html#_Q09OU1RfTUFTS19BRERSXzA_"><span class="b">CONST_MASK_ADDR</span></a><span class="f">(</span><span class="b">nr</span><span class="f">,</span> <span class="b">addr</span><span class="f">)</span>
<a name="76" /><span class="Maybe">      76:</span>             <span class="f">:</span> <span class="e">&quot;iq&quot;</span> <span class="f">(</span><span class="f">(</span><span class="b">u8</span><span class="f">)</span><a href="cpu.c_macros_ref.html#_Q09OU1RfTUFTS18w"><span class="b">CONST_MASK</span></a><span class="f">(</span><span class="b">nr</span><span class="f">)</span><span class="f">)</span>
<a name="77" /><span class="Maybe">      77:</span>             <span class="f">:</span> <span class="e">&quot;memory&quot;</span><span class="f">)</span><span class="f">;</span>
<a name="78" /><span class="Maybe">      78:</span>     <span class="f">}</span> <span class="m">else</span> <span class="f">{</span>
<a name="79" /><span class="Maybe">      79:</span>         <span class="m">asm</span> <span class="m">volatile</span><span class="f">(</span><a href="cpu.c_macros_ref.html#_TE9DS19QUkVGSVhfMA__"><span class="b">LOCK_PREFIX</span></a> <span class="e">&quot;bts %1,%0&quot;</span>
<a name="80" /><span class="Maybe">      80:</span>             <span class="f">:</span> <a href="cpu.c_macros_ref.html#_QklUT1BfQUREUl8w"><span class="b">BITOP_ADDR</span></a><span class="f">(</span><span class="b">addr</span><span class="f">)</span> <span class="f">:</span> <span class="e">&quot;Ir&quot;</span> <span class="f">(</span><span class="b">nr</span><span class="f">)</span> <span class="f">:</span> <span class="e">&quot;memory&quot;</span><span class="f">)</span><span class="f">;</span>
<a name="81" /><span class="Maybe">      81:</span>     <span class="f">}</span>
<a name="82" /><span class="Maybe">      82:</span> <span class="f">}</span>
<a name="83" /><span class="Maybe">      83:</span> 
<a name="84" /><span class="Maybe">      84:</span> <span class="k">/**</span>
<a name="85" /><span class="Maybe">      85:</span> <span class="k"> * __set_bit - Set a bit in memory</span>
<a name="86" /><span class="Maybe">      86:</span> <span class="k"> * @nr: the bit to set</span>
<a name="87" /><span class="Maybe">      87:</span> <span class="k"> * @addr: the address to start counting from</span>
<a name="88" /><span class="Maybe">      88:</span> <span class="k"> *</span>
<a name="89" /><span class="Maybe">      89:</span> <span class="k"> * Unlike set_bit(), this function is non-atomic and may be reordered.</span>
<a name="90" /><span class="Maybe">      90:</span> <span class="k"> * If it&apos;s called on the same region of memory simultaneously, the effect</span>
<a name="91" /><span class="Maybe">      91:</span> <span class="k"> * may be that only one operation succeeds.</span>
<a name="92" /><span class="Maybe">      92:</span> <span class="k"> */</span>
<a name="93" /><span class="Maybe">      93:</span> <span class="m">static</span> <span class="m">inline</span> <span class="m">void</span> <span class="b">__set_bit</span><span class="f">(</span><span class="m">long</span> <span class="b">nr</span><span class="f">,</span> <span class="m">volatile</span> <span class="m">unsigned</span> <span class="m">long</span> <span class="f">*</span><span class="b">addr</span><span class="f">)</span>
<a name="94" /><span class="Maybe">      94:</span> <span class="f">{</span>
<a name="95" /><span class="Maybe">      95:</span>     <span class="m">asm</span> <span class="m">volatile</span><span class="f">(</span><span class="e">&quot;bts %1,%0&quot;</span> <span class="f">:</span> <a href="cpu.c_macros_ref.html#_QUREUl8w"><span class="b">ADDR</span></a> <span class="f">:</span> <span class="e">&quot;Ir&quot;</span> <span class="f">(</span><span class="b">nr</span><span class="f">)</span> <span class="f">:</span> <span class="e">&quot;memory&quot;</span><span class="f">)</span><span class="f">;</span>
<a name="96" /><span class="Maybe">      96:</span> <span class="f">}</span>
<a name="97" /><span class="Maybe">      97:</span> 
<a name="98" /><span class="Maybe">      98:</span> <span class="k">/**</span>
<a name="99" /><span class="Maybe">      99:</span> <span class="k"> * clear_bit - Clears a bit in memory</span>
<a name="100" /><span class="Maybe">     100:</span> <span class="k"> * @nr: Bit to clear</span>
<a name="101" /><span class="Maybe">     101:</span> <span class="k"> * @addr: Address to start counting from</span>
<a name="102" /><span class="Maybe">     102:</span> <span class="k"> *</span>
<a name="103" /><span class="Maybe">     103:</span> <span class="k"> * clear_bit() is atomic and may not be reordered.  However, it does</span>
<a name="104" /><span class="Maybe">     104:</span> <span class="k"> * not contain a memory barrier, so if it is used for locking purposes,</span>
<a name="105" /><span class="Maybe">     105:</span> <span class="k"> * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit()</span>
<a name="106" /><span class="Maybe">     106:</span> <span class="k"> * in order to ensure changes are visible on other processors.</span>
<a name="107" /><span class="Maybe">     107:</span> <span class="k"> */</span>
<a name="108" /><span class="Maybe">     108:</span> <span class="m">static</span> <a href="cpu.c_macros_ref.html#_X19hbHdheXNfaW5saW5lXzA_"><span class="b">__always_inline</span></a> <span class="m">void</span>
<a name="109" /><span class="Maybe">     109:</span> <span class="b">clear_bit</span><span class="f">(</span><span class="m">long</span> <span class="b">nr</span><span class="f">,</span> <span class="m">volatile</span> <span class="m">unsigned</span> <span class="m">long</span> <span class="f">*</span><span class="b">addr</span><span class="f">)</span>
<a name="110" /><span class="Maybe">     110:</span> <span class="f">{</span>
<a name="111" /><span class="Maybe">     111:</span>     <span class="m">if</span> <span class="f">(</span><a href="cpu.c_macros_ref.html#_SVNfSU1NRURJQVRFXzA_"><span class="b">IS_IMMEDIATE</span></a><span class="f">(</span><span class="b">nr</span><span class="f">)</span><span class="f">)</span> <span class="f">{</span>
<a name="112" /><span class="Maybe">     112:</span>         <span class="m">asm</span> <span class="m">volatile</span><span class="f">(</span><a href="cpu.c_macros_ref.html#_TE9DS19QUkVGSVhfMA__"><span class="b">LOCK_PREFIX</span></a> <span class="e">&quot;andb %1,%0&quot;</span>
<a name="113" /><span class="Maybe">     113:</span>             <span class="f">:</span> <a href="cpu.c_macros_ref.html#_Q09OU1RfTUFTS19BRERSXzA_"><span class="b">CONST_MASK_ADDR</span></a><span class="f">(</span><span class="b">nr</span><span class="f">,</span> <span class="b">addr</span><span class="f">)</span>
<a name="114" /><span class="Maybe">     114:</span>             <span class="f">:</span> <span class="e">&quot;iq&quot;</span> <span class="f">(</span><span class="f">(</span><span class="b">u8</span><span class="f">)</span><span class="f">~</span><a href="cpu.c_macros_ref.html#_Q09OU1RfTUFTS18w"><span class="b">CONST_MASK</span></a><span class="f">(</span><span class="b">nr</span><span class="f">)</span><span class="f">)</span><span class="f">)</span><span class="f">;</span>
<a name="115" /><span class="Maybe">     115:</span>     <span class="f">}</span> <span class="m">else</span> <span class="f">{</span>
<a name="116" /><span class="Maybe">     116:</span>         <span class="m">asm</span> <span class="m">volatile</span><span class="f">(</span><a href="cpu.c_macros_ref.html#_TE9DS19QUkVGSVhfMA__"><span class="b">LOCK_PREFIX</span></a> <span class="e">&quot;btr %1,%0&quot;</span>
<a name="117" /><span class="Maybe">     117:</span>             <span class="f">:</span> <a href="cpu.c_macros_ref.html#_QklUT1BfQUREUl8w"><span class="b">BITOP_ADDR</span></a><span class="f">(</span><span class="b">addr</span><span class="f">)</span>
<a name="118" /><span class="Maybe">     118:</span>             <span class="f">:</span> <span class="e">&quot;Ir&quot;</span> <span class="f">(</span><span class="b">nr</span><span class="f">)</span><span class="f">)</span><span class="f">;</span>
<a name="119" /><span class="Maybe">     119:</span>     <span class="f">}</span>
<a name="120" /><span class="Maybe">     120:</span> <span class="f">}</span>
<a name="121" /><span class="Maybe">     121:</span> 
<a name="122" /><span class="Maybe">     122:</span> <span class="k">/*</span>
<a name="123" /><span class="Maybe">     123:</span> <span class="k"> * clear_bit_unlock - Clears a bit in memory</span>
<a name="124" /><span class="Maybe">     124:</span> <span class="k"> * @nr: Bit to clear</span>
<a name="125" /><span class="Maybe">     125:</span> <span class="k"> * @addr: Address to start counting from</span>
<a name="126" /><span class="Maybe">     126:</span> <span class="k"> *</span>
<a name="127" /><span class="Maybe">     127:</span> <span class="k"> * clear_bit() is atomic and implies release semantics before the memory</span>
<a name="128" /><span class="Maybe">     128:</span> <span class="k"> * operation. It can be used for an unlock.</span>
<a name="129" /><span class="Maybe">     129:</span> <span class="k"> */</span>
<a name="130" /><span class="Maybe">     130:</span> <span class="m">static</span> <span class="m">inline</span> <span class="m">void</span> <span class="b">clear_bit_unlock</span><span class="f">(</span><span class="m">long</span> <span class="b">nr</span><span class="f">,</span> <span class="m">volatile</span> <span class="m">unsigned</span> <span class="m">long</span> <span class="f">*</span><span class="b">addr</span><span class="f">)</span>
<a name="131" /><span class="Maybe">     131:</span> <span class="f">{</span>
<a name="132" /><span class="Maybe">     132:</span>     <a href="cpu.c_macros_ref.html#_YmFycmllcl8w"><span class="b">barrier</span></a><span class="f">(</span><span class="f">)</span><span class="f">;</span>
<a name="133" /><span class="Maybe">     133:</span>     <span class="b">clear_bit</span><span class="f">(</span><span class="b">nr</span><span class="f">,</span> <span class="b">addr</span><span class="f">)</span><span class="f">;</span>
<a name="134" /><span class="Maybe">     134:</span> <span class="f">}</span>
<a name="135" /><span class="Maybe">     135:</span> 
<a name="136" /><span class="Maybe">     136:</span> <span class="m">static</span> <span class="m">inline</span> <span class="m">void</span> <span class="b">__clear_bit</span><span class="f">(</span><span class="m">long</span> <span class="b">nr</span><span class="f">,</span> <span class="m">volatile</span> <span class="m">unsigned</span> <span class="m">long</span> <span class="f">*</span><span class="b">addr</span><span class="f">)</span>
<a name="137" /><span class="Maybe">     137:</span> <span class="f">{</span>
<a name="138" /><span class="Maybe">     138:</span>     <span class="m">asm</span> <span class="m">volatile</span><span class="f">(</span><span class="e">&quot;btr %1,%0&quot;</span> <span class="f">:</span> <a href="cpu.c_macros_ref.html#_QUREUl8w"><span class="b">ADDR</span></a> <span class="f">:</span> <span class="e">&quot;Ir&quot;</span> <span class="f">(</span><span class="b">nr</span><span class="f">)</span><span class="f">)</span><span class="f">;</span>
<a name="139" /><span class="Maybe">     139:</span> <span class="f">}</span>
<a name="140" /><span class="Maybe">     140:</span> 
<a name="141" /><span class="Maybe">     141:</span> <span class="k">/*</span>
<a name="142" /><span class="Maybe">     142:</span> <span class="k"> * __clear_bit_unlock - Clears a bit in memory</span>
<a name="143" /><span class="Maybe">     143:</span> <span class="k"> * @nr: Bit to clear</span>
<a name="144" /><span class="Maybe">     144:</span> <span class="k"> * @addr: Address to start counting from</span>
<a name="145" /><span class="Maybe">     145:</span> <span class="k"> *</span>
<a name="146" /><span class="Maybe">     146:</span> <span class="k"> * __clear_bit() is non-atomic and implies release semantics before the memory</span>
<a name="147" /><span class="Maybe">     147:</span> <span class="k"> * operation. It can be used for an unlock if no other CPUs can concurrently</span>
<a name="148" /><span class="Maybe">     148:</span> <span class="k"> * modify other bits in the word.</span>
<a name="149" /><span class="Maybe">     149:</span> <span class="k"> *</span>
<a name="150" /><span class="Maybe">     150:</span> <span class="k"> * No memory barrier is required here, because x86 cannot reorder stores past</span>
<a name="151" /><span class="Maybe">     151:</span> <span class="k"> * older loads. Same principle as spin_unlock.</span>
<a name="152" /><span class="Maybe">     152:</span> <span class="k"> */</span>
<a name="153" /><span class="Maybe">     153:</span> <span class="m">static</span> <span class="m">inline</span> <span class="m">void</span> <span class="b">__clear_bit_unlock</span><span class="f">(</span><span class="m">long</span> <span class="b">nr</span><span class="f">,</span> <span class="m">volatile</span> <span class="m">unsigned</span> <span class="m">long</span> <span class="f">*</span><span class="b">addr</span><span class="f">)</span>
<a name="154" /><span class="Maybe">     154:</span> <span class="f">{</span>
<a name="155" /><span class="Maybe">     155:</span>     <a href="cpu.c_macros_ref.html#_YmFycmllcl8w"><span class="b">barrier</span></a><span class="f">(</span><span class="f">)</span><span class="f">;</span>
<a name="156" /><span class="Maybe">     156:</span>     <span class="b">__clear_bit</span><span class="f">(</span><span class="b">nr</span><span class="f">,</span> <span class="b">addr</span><span class="f">)</span><span class="f">;</span>
<a name="157" /><span class="Maybe">     157:</span> <span class="f">}</span>
<a name="158" /><span class="Maybe">     158:</span> 
<a name="159" /><span class="Maybe">     159:</span> <span class="f">#</span><span class="n">define</span> <a href="cpu.c_macros_noref.html#_c21wX21iX19iZWZvcmVfY2xlYXJfYml0XzA_"><span class="b">smp_mb__before_clear_bit</span></a><span class="f">(</span><span class="f">)</span>    <a href="cpu.c_macros_ref.html#_YmFycmllcl8w"><span class="b">barrier</span></a><span class="f">(</span><span class="f">)</span>
<a name="160" /><span class="Maybe">     160:</span> <span class="f">#</span><span class="n">define</span> <a href="cpu.c_macros_noref.html#_c21wX21iX19hZnRlcl9jbGVhcl9iaXRfMA__"><span class="b">smp_mb__after_clear_bit</span></a><span class="f">(</span><span class="f">)</span>    <a href="cpu.c_macros_ref.html#_YmFycmllcl8w"><span class="b">barrier</span></a><span class="f">(</span><span class="f">)</span>
<a name="161" /><span class="Maybe">     161:</span> 
<a name="162" /><span class="Maybe">     162:</span> <span class="k">/**</span>
<a name="163" /><span class="Maybe">     163:</span> <span class="k"> * __change_bit - Toggle a bit in memory</span>
<a name="164" /><span class="Maybe">     164:</span> <span class="k"> * @nr: the bit to change</span>
<a name="165" /><span class="Maybe">     165:</span> <span class="k"> * @addr: the address to start counting from</span>
<a name="166" /><span class="Maybe">     166:</span> <span class="k"> *</span>
<a name="167" /><span class="Maybe">     167:</span> <span class="k"> * Unlike change_bit(), this function is non-atomic and may be reordered.</span>
<a name="168" /><span class="Maybe">     168:</span> <span class="k"> * If it&apos;s called on the same region of memory simultaneously, the effect</span>
<a name="169" /><span class="Maybe">     169:</span> <span class="k"> * may be that only one operation succeeds.</span>
<a name="170" /><span class="Maybe">     170:</span> <span class="k"> */</span>
<a name="171" /><span class="Maybe">     171:</span> <span class="m">static</span> <span class="m">inline</span> <span class="m">void</span> <span class="b">__change_bit</span><span class="f">(</span><span class="m">long</span> <span class="b">nr</span><span class="f">,</span> <span class="m">volatile</span> <span class="m">unsigned</span> <span class="m">long</span> <span class="f">*</span><span class="b">addr</span><span class="f">)</span>
<a name="172" /><span class="Maybe">     172:</span> <span class="f">{</span>
<a name="173" /><span class="Maybe">     173:</span>     <span class="m">asm</span> <span class="m">volatile</span><span class="f">(</span><span class="e">&quot;btc %1,%0&quot;</span> <span class="f">:</span> <a href="cpu.c_macros_ref.html#_QUREUl8w"><span class="b">ADDR</span></a> <span class="f">:</span> <span class="e">&quot;Ir&quot;</span> <span class="f">(</span><span class="b">nr</span><span class="f">)</span><span class="f">)</span><span class="f">;</span>
<a name="174" /><span class="Maybe">     174:</span> <span class="f">}</span>
<a name="175" /><span class="Maybe">     175:</span> 
<a name="176" /><span class="Maybe">     176:</span> <span class="k">/**</span>
<a name="177" /><span class="Maybe">     177:</span> <span class="k"> * change_bit - Toggle a bit in memory</span>
<a name="178" /><span class="Maybe">     178:</span> <span class="k"> * @nr: Bit to change</span>
<a name="179" /><span class="Maybe">     179:</span> <span class="k"> * @addr: Address to start counting from</span>
<a name="180" /><span class="Maybe">     180:</span> <span class="k"> *</span>
<a name="181" /><span class="Maybe">     181:</span> <span class="k"> * change_bit() is atomic and may not be reordered.</span>
<a name="182" /><span class="Maybe">     182:</span> <span class="k"> * Note that @nr may be almost arbitrarily large; this function is not</span>
<a name="183" /><span class="Maybe">     183:</span> <span class="k"> * restricted to acting on a single-word quantity.</span>
<a name="184" /><span class="Maybe">     184:</span> <span class="k"> */</span>
<a name="185" /><span class="Maybe">     185:</span> <span class="m">static</span> <span class="m">inline</span> <span class="m">void</span> <span class="b">change_bit</span><span class="f">(</span><span class="m">long</span> <span class="b">nr</span><span class="f">,</span> <span class="m">volatile</span> <span class="m">unsigned</span> <span class="m">long</span> <span class="f">*</span><span class="b">addr</span><span class="f">)</span>
<a name="186" /><span class="Maybe">     186:</span> <span class="f">{</span>
<a name="187" /><span class="Maybe">     187:</span>     <span class="m">if</span> <span class="f">(</span><a href="cpu.c_macros_ref.html#_SVNfSU1NRURJQVRFXzA_"><span class="b">IS_IMMEDIATE</span></a><span class="f">(</span><span class="b">nr</span><span class="f">)</span><span class="f">)</span> <span class="f">{</span>
<a name="188" /><span class="Maybe">     188:</span>         <span class="m">asm</span> <span class="m">volatile</span><span class="f">(</span><a href="cpu.c_macros_ref.html#_TE9DS19QUkVGSVhfMA__"><span class="b">LOCK_PREFIX</span></a> <span class="e">&quot;xorb %1,%0&quot;</span>
<a name="189" /><span class="Maybe">     189:</span>             <span class="f">:</span> <a href="cpu.c_macros_ref.html#_Q09OU1RfTUFTS19BRERSXzA_"><span class="b">CONST_MASK_ADDR</span></a><span class="f">(</span><span class="b">nr</span><span class="f">,</span> <span class="b">addr</span><span class="f">)</span>
<a name="190" /><span class="Maybe">     190:</span>             <span class="f">:</span> <span class="e">&quot;iq&quot;</span> <span class="f">(</span><span class="f">(</span><span class="b">u8</span><span class="f">)</span><a href="cpu.c_macros_ref.html#_Q09OU1RfTUFTS18w"><span class="b">CONST_MASK</span></a><span class="f">(</span><span class="b">nr</span><span class="f">)</span><span class="f">)</span><span class="f">)</span><span class="f">;</span>
<a name="191" /><span class="Maybe">     191:</span>     <span class="f">}</span> <span class="m">else</span> <span class="f">{</span>
<a name="192" /><span class="Maybe">     192:</span>         <span class="m">asm</span> <span class="m">volatile</span><span class="f">(</span><a href="cpu.c_macros_ref.html#_TE9DS19QUkVGSVhfMA__"><span class="b">LOCK_PREFIX</span></a> <span class="e">&quot;btc %1,%0&quot;</span>
<a name="193" /><span class="Maybe">     193:</span>             <span class="f">:</span> <a href="cpu.c_macros_ref.html#_QklUT1BfQUREUl8w"><span class="b">BITOP_ADDR</span></a><span class="f">(</span><span class="b">addr</span><span class="f">)</span>
<a name="194" /><span class="Maybe">     194:</span>             <span class="f">:</span> <span class="e">&quot;Ir&quot;</span> <span class="f">(</span><span class="b">nr</span><span class="f">)</span><span class="f">)</span><span class="f">;</span>
<a name="195" /><span class="Maybe">     195:</span>     <span class="f">}</span>
<a name="196" /><span class="Maybe">     196:</span> <span class="f">}</span>
<a name="197" /><span class="Maybe">     197:</span> 
<a name="198" /><span class="Maybe">     198:</span> <span class="k">/**</span>
<a name="199" /><span class="Maybe">     199:</span> <span class="k"> * test_and_set_bit - Set a bit and return its old value</span>
<a name="200" /><span class="Maybe">     200:</span> <span class="k"> * @nr: Bit to set</span>
<a name="201" /><span class="Maybe">     201:</span> <span class="k"> * @addr: Address to count from</span>
<a name="202" /><span class="Maybe">     202:</span> <span class="k"> *</span>
<a name="203" /><span class="Maybe">     203:</span> <span class="k"> * This operation is atomic and cannot be reordered.</span>
<a name="204" /><span class="Maybe">     204:</span> <span class="k"> * It also implies a memory barrier.</span>
<a name="205" /><span class="Maybe">     205:</span> <span class="k"> */</span>
<a name="206" /><span class="Maybe">     206:</span> <span class="m">static</span> <span class="m">inline</span> <span class="m">int</span> <span class="b">test_and_set_bit</span><span class="f">(</span><span class="m">long</span> <span class="b">nr</span><span class="f">,</span> <span class="m">volatile</span> <span class="m">unsigned</span> <span class="m">long</span> <span class="f">*</span><span class="b">addr</span><span class="f">)</span>
<a name="207" /><span class="Maybe">     207:</span> <span class="f">{</span>
<a name="208" /><span class="Maybe">     208:</span>     <a href="cpu.c_macros_ref.html#_R0VOX0JJTkFSWV9STVdjY18w"><span class="b">GEN_BINARY_RMWcc</span></a><span class="f">(</span><a href="cpu.c_macros_ref.html#_TE9DS19QUkVGSVhfMA__"><span class="b">LOCK_PREFIX</span></a> <span class="e">&quot;bts&quot;</span><span class="f">,</span> <span class="f">*</span><span class="b">addr</span><span class="f">,</span> <span class="e">&quot;Ir&quot;</span><span class="f">,</span> <span class="b">nr</span><span class="f">,</span> <span class="e">&quot;%0&quot;</span><span class="f">,</span> <span class="e">&quot;c&quot;</span><span class="f">)</span><span class="f">;</span>
<a name="209" /><span class="Maybe">     209:</span> <span class="f">}</span>
<a name="210" /><span class="Maybe">     210:</span> 
<a name="211" /><span class="Maybe">     211:</span> <span class="k">/**</span>
<a name="212" /><span class="Maybe">     212:</span> <span class="k"> * test_and_set_bit_lock - Set a bit and return its old value for lock</span>
<a name="213" /><span class="Maybe">     213:</span> <span class="k"> * @nr: Bit to set</span>
<a name="214" /><span class="Maybe">     214:</span> <span class="k"> * @addr: Address to count from</span>
<a name="215" /><span class="Maybe">     215:</span> <span class="k"> *</span>
<a name="216" /><span class="Maybe">     216:</span> <span class="k"> * This is the same as test_and_set_bit on x86.</span>
<a name="217" /><span class="Maybe">     217:</span> <span class="k"> */</span>
<a name="218" /><span class="Maybe">     218:</span> <span class="m">static</span> <a href="cpu.c_macros_ref.html#_X19hbHdheXNfaW5saW5lXzA_"><span class="b">__always_inline</span></a> <span class="m">int</span>
<a name="219" /><span class="Maybe">     219:</span> <span class="b">test_and_set_bit_lock</span><span class="f">(</span><span class="m">long</span> <span class="b">nr</span><span class="f">,</span> <span class="m">volatile</span> <span class="m">unsigned</span> <span class="m">long</span> <span class="f">*</span><span class="b">addr</span><span class="f">)</span>
<a name="220" /><span class="Maybe">     220:</span> <span class="f">{</span>
<a name="221" /><span class="Maybe">     221:</span>     <span class="m">return</span> <span class="b">test_and_set_bit</span><span class="f">(</span><span class="b">nr</span><span class="f">,</span> <span class="b">addr</span><span class="f">)</span><span class="f">;</span>
<a name="222" /><span class="Maybe">     222:</span> <span class="f">}</span>
<a name="223" /><span class="Maybe">     223:</span> 
<a name="224" /><span class="Maybe">     224:</span> <span class="k">/**</span>
<a name="225" /><span class="Maybe">     225:</span> <span class="k"> * __test_and_set_bit - Set a bit and return its old value</span>
<a name="226" /><span class="Maybe">     226:</span> <span class="k"> * @nr: Bit to set</span>
<a name="227" /><span class="Maybe">     227:</span> <span class="k"> * @addr: Address to count from</span>
<a name="228" /><span class="Maybe">     228:</span> <span class="k"> *</span>
<a name="229" /><span class="Maybe">     229:</span> <span class="k"> * This operation is non-atomic and can be reordered.</span>
<a name="230" /><span class="Maybe">     230:</span> <span class="k"> * If two examples of this operation race, one can appear to succeed</span>
<a name="231" /><span class="Maybe">     231:</span> <span class="k"> * but actually fail.  You must protect multiple accesses with a lock.</span>
<a name="232" /><span class="Maybe">     232:</span> <span class="k"> */</span>
<a name="233" /><span class="Maybe">     233:</span> <span class="m">static</span> <span class="m">inline</span> <span class="m">int</span> <span class="b">__test_and_set_bit</span><span class="f">(</span><span class="m">long</span> <span class="b">nr</span><span class="f">,</span> <span class="m">volatile</span> <span class="m">unsigned</span> <span class="m">long</span> <span class="f">*</span><span class="b">addr</span><span class="f">)</span>
<a name="234" /><span class="Maybe">     234:</span> <span class="f">{</span>
<a name="235" /><span class="Maybe">     235:</span>     <span class="m">int</span> <span class="b">oldbit</span><span class="f">;</span>
<a name="236" /><span class="Maybe">     236:</span> 
<a name="237" /><span class="Maybe">     237:</span>     <span class="m">asm</span><span class="f">(</span><span class="e">&quot;bts %2,%1\n\t&quot;</span>
<a name="238" /><span class="Maybe">     238:</span>         <span class="e">&quot;sbb %0,%0&quot;</span>
<a name="239" /><span class="Maybe">     239:</span>         <span class="f">:</span> <span class="e">&quot;=r&quot;</span> <span class="f">(</span><span class="b">oldbit</span><span class="f">)</span><span class="f">,</span> <a href="cpu.c_macros_ref.html#_QUREUl8w"><span class="b">ADDR</span></a>
<a name="240" /><span class="Maybe">     240:</span>         <span class="f">:</span> <span class="e">&quot;Ir&quot;</span> <span class="f">(</span><span class="b">nr</span><span class="f">)</span><span class="f">)</span><span class="f">;</span>
<a name="241" /><span class="Maybe">     241:</span>     <span class="m">return</span> <span class="b">oldbit</span><span class="f">;</span>
<a name="242" /><span class="Maybe">     242:</span> <span class="f">}</span>
<a name="243" /><span class="Maybe">     243:</span> 
<a name="244" /><span class="Maybe">     244:</span> <span class="k">/**</span>
<a name="245" /><span class="Maybe">     245:</span> <span class="k"> * test_and_clear_bit - Clear a bit and return its old value</span>
<a name="246" /><span class="Maybe">     246:</span> <span class="k"> * @nr: Bit to clear</span>
<a name="247" /><span class="Maybe">     247:</span> <span class="k"> * @addr: Address to count from</span>
<a name="248" /><span class="Maybe">     248:</span> <span class="k"> *</span>
<a name="249" /><span class="Maybe">     249:</span> <span class="k"> * This operation is atomic and cannot be reordered.</span>
<a name="250" /><span class="Maybe">     250:</span> <span class="k"> * It also implies a memory barrier.</span>
<a name="251" /><span class="Maybe">     251:</span> <span class="k"> */</span>
<a name="252" /><span class="Maybe">     252:</span> <span class="m">static</span> <span class="m">inline</span> <span class="m">int</span> <span class="b">test_and_clear_bit</span><span class="f">(</span><span class="m">long</span> <span class="b">nr</span><span class="f">,</span> <span class="m">volatile</span> <span class="m">unsigned</span> <span class="m">long</span> <span class="f">*</span><span class="b">addr</span><span class="f">)</span>
<a name="253" /><span class="Maybe">     253:</span> <span class="f">{</span>
<a name="254" /><span class="Maybe">     254:</span>     <a href="cpu.c_macros_ref.html#_R0VOX0JJTkFSWV9STVdjY18w"><span class="b">GEN_BINARY_RMWcc</span></a><span class="f">(</span><a href="cpu.c_macros_ref.html#_TE9DS19QUkVGSVhfMA__"><span class="b">LOCK_PREFIX</span></a> <span class="e">&quot;btr&quot;</span><span class="f">,</span> <span class="f">*</span><span class="b">addr</span><span class="f">,</span> <span class="e">&quot;Ir&quot;</span><span class="f">,</span> <span class="b">nr</span><span class="f">,</span> <span class="e">&quot;%0&quot;</span><span class="f">,</span> <span class="e">&quot;c&quot;</span><span class="f">)</span><span class="f">;</span>
<a name="255" /><span class="Maybe">     255:</span> <span class="f">}</span>
<a name="256" /><span class="Maybe">     256:</span> 
<a name="257" /><span class="Maybe">     257:</span> <span class="k">/**</span>
<a name="258" /><span class="Maybe">     258:</span> <span class="k"> * __test_and_clear_bit - Clear a bit and return its old value</span>
<a name="259" /><span class="Maybe">     259:</span> <span class="k"> * @nr: Bit to clear</span>
<a name="260" /><span class="Maybe">     260:</span> <span class="k"> * @addr: Address to count from</span>
<a name="261" /><span class="Maybe">     261:</span> <span class="k"> *</span>
<a name="262" /><span class="Maybe">     262:</span> <span class="k"> * This operation is non-atomic and can be reordered.</span>
<a name="263" /><span class="Maybe">     263:</span> <span class="k"> * If two examples of this operation race, one can appear to succeed</span>
<a name="264" /><span class="Maybe">     264:</span> <span class="k"> * but actually fail.  You must protect multiple accesses with a lock.</span>
<a name="265" /><span class="Maybe">     265:</span> <span class="k"> *</span>
<a name="266" /><span class="Maybe">     266:</span> <span class="k"> * Note: the operation is performed atomically with respect to</span>
<a name="267" /><span class="Maybe">     267:</span> <span class="k"> * the local CPU, but not other CPUs. Portable code should not</span>
<a name="268" /><span class="Maybe">     268:</span> <span class="k"> * rely on this behaviour.</span>
<a name="269" /><span class="Maybe">     269:</span> <span class="k"> * KVM relies on this behaviour on x86 for modifying memory that is also</span>
<a name="270" /><span class="Maybe">     270:</span> <span class="k"> * accessed from a hypervisor on the same CPU if running in a VM: don&apos;t change</span>
<a name="271" /><span class="Maybe">     271:</span> <span class="k"> * this without also updating arch/x86/kernel/kvm.c</span>
<a name="272" /><span class="Maybe">     272:</span> <span class="k"> */</span>
<a name="273" /><span class="Maybe">     273:</span> <span class="m">static</span> <span class="m">inline</span> <span class="m">int</span> <span class="b">__test_and_clear_bit</span><span class="f">(</span><span class="m">long</span> <span class="b">nr</span><span class="f">,</span> <span class="m">volatile</span> <span class="m">unsigned</span> <span class="m">long</span> <span class="f">*</span><span class="b">addr</span><span class="f">)</span>
<a name="274" /><span class="Maybe">     274:</span> <span class="f">{</span>
<a name="275" /><span class="Maybe">     275:</span>     <span class="m">int</span> <span class="b">oldbit</span><span class="f">;</span>
<a name="276" /><span class="Maybe">     276:</span> 
<a name="277" /><span class="Maybe">     277:</span>     <span class="m">asm</span> <span class="m">volatile</span><span class="f">(</span><span class="e">&quot;btr %2,%1\n\t&quot;</span>
<a name="278" /><span class="Maybe">     278:</span>              <span class="e">&quot;sbb %0,%0&quot;</span>
<a name="279" /><span class="Maybe">     279:</span>              <span class="f">:</span> <span class="e">&quot;=r&quot;</span> <span class="f">(</span><span class="b">oldbit</span><span class="f">)</span><span class="f">,</span> <a href="cpu.c_macros_ref.html#_QUREUl8w"><span class="b">ADDR</span></a>
<a name="280" /><span class="Maybe">     280:</span>              <span class="f">:</span> <span class="e">&quot;Ir&quot;</span> <span class="f">(</span><span class="b">nr</span><span class="f">)</span><span class="f">)</span><span class="f">;</span>
<a name="281" /><span class="Maybe">     281:</span>     <span class="m">return</span> <span class="b">oldbit</span><span class="f">;</span>
<a name="282" /><span class="Maybe">     282:</span> <span class="f">}</span>
<a name="283" /><span class="Maybe">     283:</span> 
<a name="284" /><span class="Maybe">     284:</span> <span class="k">/* WARNING: non atomic and it can be reordered! */</span>
<a name="285" /><span class="Maybe">     285:</span> <span class="m">static</span> <span class="m">inline</span> <span class="m">int</span> <span class="b">__test_and_change_bit</span><span class="f">(</span><span class="m">long</span> <span class="b">nr</span><span class="f">,</span> <span class="m">volatile</span> <span class="m">unsigned</span> <span class="m">long</span> <span class="f">*</span><span class="b">addr</span><span class="f">)</span>
<a name="286" /><span class="Maybe">     286:</span> <span class="f">{</span>
<a name="287" /><span class="Maybe">     287:</span>     <span class="m">int</span> <span class="b">oldbit</span><span class="f">;</span>
<a name="288" /><span class="Maybe">     288:</span> 
<a name="289" /><span class="Maybe">     289:</span>     <span class="m">asm</span> <span class="m">volatile</span><span class="f">(</span><span class="e">&quot;btc %2,%1\n\t&quot;</span>
<a name="290" /><span class="Maybe">     290:</span>              <span class="e">&quot;sbb %0,%0&quot;</span>
<a name="291" /><span class="Maybe">     291:</span>              <span class="f">:</span> <span class="e">&quot;=r&quot;</span> <span class="f">(</span><span class="b">oldbit</span><span class="f">)</span><span class="f">,</span> <a href="cpu.c_macros_ref.html#_QUREUl8w"><span class="b">ADDR</span></a>
<a name="292" /><span class="Maybe">     292:</span>              <span class="f">:</span> <span class="e">&quot;Ir&quot;</span> <span class="f">(</span><span class="b">nr</span><span class="f">)</span> <span class="f">:</span> <span class="e">&quot;memory&quot;</span><span class="f">)</span><span class="f">;</span>
<a name="293" /><span class="Maybe">     293:</span> 
<a name="294" /><span class="Maybe">     294:</span>     <span class="m">return</span> <span class="b">oldbit</span><span class="f">;</span>
<a name="295" /><span class="Maybe">     295:</span> <span class="f">}</span>
<a name="296" /><span class="Maybe">     296:</span> 
<a name="297" /><span class="Maybe">     297:</span> <span class="k">/**</span>
<a name="298" /><span class="Maybe">     298:</span> <span class="k"> * test_and_change_bit - Change a bit and return its old value</span>
<a name="299" /><span class="Maybe">     299:</span> <span class="k"> * @nr: Bit to change</span>
<a name="300" /><span class="Maybe">     300:</span> <span class="k"> * @addr: Address to count from</span>
<a name="301" /><span class="Maybe">     301:</span> <span class="k"> *</span>
<a name="302" /><span class="Maybe">     302:</span> <span class="k"> * This operation is atomic and cannot be reordered.</span>
<a name="303" /><span class="Maybe">     303:</span> <span class="k"> * It also implies a memory barrier.</span>
<a name="304" /><span class="Maybe">     304:</span> <span class="k"> */</span>
<a name="305" /><span class="Maybe">     305:</span> <span class="m">static</span> <span class="m">inline</span> <span class="m">int</span> <span class="b">test_and_change_bit</span><span class="f">(</span><span class="m">long</span> <span class="b">nr</span><span class="f">,</span> <span class="m">volatile</span> <span class="m">unsigned</span> <span class="m">long</span> <span class="f">*</span><span class="b">addr</span><span class="f">)</span>
<a name="306" /><span class="Maybe">     306:</span> <span class="f">{</span>
<a name="307" /><span class="Maybe">     307:</span>     <a href="cpu.c_macros_ref.html#_R0VOX0JJTkFSWV9STVdjY18w"><span class="b">GEN_BINARY_RMWcc</span></a><span class="f">(</span><a href="cpu.c_macros_ref.html#_TE9DS19QUkVGSVhfMA__"><span class="b">LOCK_PREFIX</span></a> <span class="e">&quot;btc&quot;</span><span class="f">,</span> <span class="f">*</span><span class="b">addr</span><span class="f">,</span> <span class="e">&quot;Ir&quot;</span><span class="f">,</span> <span class="b">nr</span><span class="f">,</span> <span class="e">&quot;%0&quot;</span><span class="f">,</span> <span class="e">&quot;c&quot;</span><span class="f">)</span><span class="f">;</span>
<a name="308" /><span class="Maybe">     308:</span> <span class="f">}</span>
<a name="309" /><span class="Maybe">     309:</span> 
<a name="310" /><span class="Maybe">     310:</span> <span class="m">static</span> <a href="cpu.c_macros_ref.html#_X19hbHdheXNfaW5saW5lXzA_"><span class="b">__always_inline</span></a> <span class="m">int</span> <span class="b">constant_test_bit</span><span class="f">(</span><span class="m">long</span> <span class="b">nr</span><span class="f">,</span> <span class="m">const</span> <span class="m">volatile</span> <span class="m">unsigned</span> <span class="m">long</span> <span class="f">*</span><span class="b">addr</span><span class="f">)</span>
<a name="311" /><span class="Maybe">     311:</span> <span class="f">{</span>
<a name="312" /><span class="Maybe">     312:</span>     <span class="m">return</span> <span class="f">(</span><span class="f">(</span><span class="c">1UL</span> <span class="f">&lt;&lt;</span> <span class="f">(</span><span class="b">nr</span> <span class="f">&amp;</span> <span class="f">(</span><a href="cpu.c_macros_ref.html#_QklUU19QRVJfTE9OR18w"><span class="b">BITS_PER_LONG</span></a><span class="f">-</span><span class="c">1</span><span class="f">)</span><span class="f">)</span><span class="f">)</span> <span class="f">&amp;</span>
<a name="313" /><span class="Maybe">     313:</span>         <span class="f">(</span><span class="b">addr</span><span class="f">[</span><span class="b">nr</span> <span class="f">&gt;&gt;</span> <a href="cpu.c_macros_ref.html#_X0JJVE9QU19MT05HX1NISUZUXzA_"><span class="b">_BITOPS_LONG_SHIFT</span></a><span class="f">]</span><span class="f">)</span><span class="f">)</span> <span class="f">!=</span> <span class="c">0</span><span class="f">;</span>
<a name="314" /><span class="Maybe">     314:</span> <span class="f">}</span>
<a name="315" /><span class="Maybe">     315:</span> 
<a name="316" /><span class="Maybe">     316:</span> <span class="m">static</span> <span class="m">inline</span> <span class="m">int</span> <span class="b">variable_test_bit</span><span class="f">(</span><span class="m">long</span> <span class="b">nr</span><span class="f">,</span> <span class="m">volatile</span> <span class="m">const</span> <span class="m">unsigned</span> <span class="m">long</span> <span class="f">*</span><span class="b">addr</span><span class="f">)</span>
<a name="317" /><span class="Maybe">     317:</span> <span class="f">{</span>
<a name="318" /><span class="Maybe">     318:</span>     <span class="m">int</span> <span class="b">oldbit</span><span class="f">;</span>
<a name="319" /><span class="Maybe">     319:</span> 
<a name="320" /><span class="Maybe">     320:</span>     <span class="m">asm</span> <span class="m">volatile</span><span class="f">(</span><span class="e">&quot;bt %2,%1\n\t&quot;</span>
<a name="321" /><span class="Maybe">     321:</span>              <span class="e">&quot;sbb %0,%0&quot;</span>
<a name="322" /><span class="Maybe">     322:</span>              <span class="f">:</span> <span class="e">&quot;=r&quot;</span> <span class="f">(</span><span class="b">oldbit</span><span class="f">)</span>
<a name="323" /><span class="Maybe">     323:</span>              <span class="f">:</span> <span class="e">&quot;m&quot;</span> <span class="f">(</span><span class="f">*</span><span class="f">(</span><span class="m">unsigned</span> <span class="m">long</span> <span class="f">*</span><span class="f">)</span><span class="b">addr</span><span class="f">)</span><span class="f">,</span> <span class="e">&quot;Ir&quot;</span> <span class="f">(</span><span class="b">nr</span><span class="f">)</span><span class="f">)</span><span class="f">;</span>
<a name="324" /><span class="Maybe">     324:</span> 
<a name="325" /><span class="Maybe">     325:</span>     <span class="m">return</span> <span class="b">oldbit</span><span class="f">;</span>
<a name="326" /><span class="Maybe">     326:</span> <span class="f">}</span>
<a name="327" /><span class="Maybe">     327:</span> 
<a name="328" /><span class="False">     328:</span> <span class="f">#</span><span class="n">if</span> <span class="c">0</span> <span class="k">/* Fool kernel-doc since it doesn&apos;t do macros yet */</span>
<a name="329" /><span class="False">     329:</span> <span class="k">/**</span>
<a name="330" /><span class="False">     330:</span> <span class="k"> * test_bit - Determine whether a bit is set</span>
<a name="331" /><span class="False">     331:</span> <span class="k"> * @nr: bit number to test</span>
<a name="332" /><span class="False">     332:</span> <span class="k"> * @addr: Address to start counting from</span>
<a name="333" /><span class="False">     333:</span> <span class="k"> */</span>
<a name="334" /><span class="False">     334:</span> <span class="m">static</span> <span class="m">int</span> <a href="cpu.c_macros_ref.html#_dGVzdF9iaXRfMA__"><span class="b">test_bit</span></a><span class="f">(</span><span class="m">int</span> <span class="b">nr</span><span class="f">,</span> <span class="m">const</span> <span class="m">volatile</span> <span class="m">unsigned</span> <span class="m">long</span> <span class="f">*</span><span class="b">addr</span><span class="f">)</span><span class="f">;</span>
<a name="335" /><span class="Maybe">     335:</span> <span class="f">#</span><span class="n">endif</span>
<a name="336" /><span class="Maybe">     336:</span> 
<a name="337" /><span class="Maybe">     337:</span> <span class="f">#</span><span class="n">define</span> <a href="cpu.c_macros_ref.html#_dGVzdF9iaXRfMA__"><span class="b">test_bit</span></a><span class="f">(</span><span class="b">nr</span><span class="f">,</span> <span class="b">addr</span><span class="f">)</span>            \
<a name="338" /><span class="Maybe">     338:</span>     <span class="f">(</span><span class="b">__builtin_constant_p</span><span class="f">(</span><span class="f">(</span><span class="b">nr</span><span class="f">)</span><span class="f">)</span>        \
<a name="339" /><span class="Maybe">     339:</span>      <span class="f">?</span> <span class="b">constant_test_bit</span><span class="f">(</span><span class="f">(</span><span class="b">nr</span><span class="f">)</span><span class="f">,</span> <span class="f">(</span><span class="b">addr</span><span class="f">)</span><span class="f">)</span>    \
<a name="340" /><span class="Maybe">     340:</span>      <span class="f">:</span> <span class="b">variable_test_bit</span><span class="f">(</span><span class="f">(</span><span class="b">nr</span><span class="f">)</span><span class="f">,</span> <span class="f">(</span><span class="b">addr</span><span class="f">)</span><span class="f">)</span><span class="f">)</span>
<a name="341" /><span class="Maybe">     341:</span> 
<a name="342" /><span class="Maybe">     342:</span> <span class="k">/**</span>
<a name="343" /><span class="Maybe">     343:</span> <span class="k"> * __ffs - find first set bit in word</span>
<a name="344" /><span class="Maybe">     344:</span> <span class="k"> * @word: The word to search</span>
<a name="345" /><span class="Maybe">     345:</span> <span class="k"> *</span>
<a name="346" /><span class="Maybe">     346:</span> <span class="k"> * Undefined if no bit exists, so code should check against 0 first.</span>
<a name="347" /><span class="Maybe">     347:</span> <span class="k"> */</span>
<a name="348" /><span class="Maybe">     348:</span> <span class="m">static</span> <span class="m">inline</span> <span class="m">unsigned</span> <span class="m">long</span> <span class="b">__ffs</span><span class="f">(</span><span class="m">unsigned</span> <span class="m">long</span> <span class="b">word</span><span class="f">)</span>
<a name="349" /><span class="Maybe">     349:</span> <span class="f">{</span>
<a name="350" /><span class="Maybe">     350:</span>     <span class="m">asm</span><span class="f">(</span><span class="e">&quot;rep; bsf %1,%0&quot;</span>
<a name="351" /><span class="Maybe">     351:</span>         <span class="f">:</span> <span class="e">&quot;=r&quot;</span> <span class="f">(</span><span class="b">word</span><span class="f">)</span>
<a name="352" /><span class="Maybe">     352:</span>         <span class="f">:</span> <span class="e">&quot;rm&quot;</span> <span class="f">(</span><span class="b">word</span><span class="f">)</span><span class="f">)</span><span class="f">;</span>
<a name="353" /><span class="Maybe">     353:</span>     <span class="m">return</span> <span class="b">word</span><span class="f">;</span>
<a name="354" /><span class="Maybe">     354:</span> <span class="f">}</span>
<a name="355" /><span class="Maybe">     355:</span> 
<a name="356" /><span class="Maybe">     356:</span> <span class="k">/**</span>
<a name="357" /><span class="Maybe">     357:</span> <span class="k"> * ffz - find first zero bit in word</span>
<a name="358" /><span class="Maybe">     358:</span> <span class="k"> * @word: The word to search</span>
<a name="359" /><span class="Maybe">     359:</span> <span class="k"> *</span>
<a name="360" /><span class="Maybe">     360:</span> <span class="k"> * Undefined if no zero exists, so code should check against ~0UL first.</span>
<a name="361" /><span class="Maybe">     361:</span> <span class="k"> */</span>
<a name="362" /><span class="Maybe">     362:</span> <span class="m">static</span> <span class="m">inline</span> <span class="m">unsigned</span> <span class="m">long</span> <span class="b">ffz</span><span class="f">(</span><span class="m">unsigned</span> <span class="m">long</span> <span class="b">word</span><span class="f">)</span>
<a name="363" /><span class="Maybe">     363:</span> <span class="f">{</span>
<a name="364" /><span class="Maybe">     364:</span>     <span class="m">asm</span><span class="f">(</span><span class="e">&quot;rep; bsf %1,%0&quot;</span>
<a name="365" /><span class="Maybe">     365:</span>         <span class="f">:</span> <span class="e">&quot;=r&quot;</span> <span class="f">(</span><span class="b">word</span><span class="f">)</span>
<a name="366" /><span class="Maybe">     366:</span>         <span class="f">:</span> <span class="e">&quot;r&quot;</span> <span class="f">(</span><span class="f">~</span><span class="b">word</span><span class="f">)</span><span class="f">)</span><span class="f">;</span>
<a name="367" /><span class="Maybe">     367:</span>     <span class="m">return</span> <span class="b">word</span><span class="f">;</span>
<a name="368" /><span class="Maybe">     368:</span> <span class="f">}</span>
<a name="369" /><span class="Maybe">     369:</span> 
<a name="370" /><span class="Maybe">     370:</span> <span class="k">/*</span>
<a name="371" /><span class="Maybe">     371:</span> <span class="k"> * __fls: find last set bit in word</span>
<a name="372" /><span class="Maybe">     372:</span> <span class="k"> * @word: The word to search</span>
<a name="373" /><span class="Maybe">     373:</span> <span class="k"> *</span>
<a name="374" /><span class="Maybe">     374:</span> <span class="k"> * Undefined if no set bit exists, so code should check against 0 first.</span>
<a name="375" /><span class="Maybe">     375:</span> <span class="k"> */</span>
<a name="376" /><span class="Maybe">     376:</span> <span class="m">static</span> <span class="m">inline</span> <span class="m">unsigned</span> <span class="m">long</span> <span class="b">__fls</span><span class="f">(</span><span class="m">unsigned</span> <span class="m">long</span> <span class="b">word</span><span class="f">)</span>
<a name="377" /><span class="Maybe">     377:</span> <span class="f">{</span>
<a name="378" /><span class="Maybe">     378:</span>     <span class="m">asm</span><span class="f">(</span><span class="e">&quot;bsr %1,%0&quot;</span>
<a name="379" /><span class="Maybe">     379:</span>         <span class="f">:</span> <span class="e">&quot;=r&quot;</span> <span class="f">(</span><span class="b">word</span><span class="f">)</span>
<a name="380" /><span class="Maybe">     380:</span>         <span class="f">:</span> <span class="e">&quot;rm&quot;</span> <span class="f">(</span><span class="b">word</span><span class="f">)</span><span class="f">)</span><span class="f">;</span>
<a name="381" /><span class="Maybe">     381:</span>     <span class="m">return</span> <span class="b">word</span><span class="f">;</span>
<a name="382" /><span class="Maybe">     382:</span> <span class="f">}</span>
<a name="383" /><span class="Maybe">     383:</span> 
<a name="384" /><span class="Maybe">     384:</span> <span class="f">#</span><span class="n">undef</span> <a href="cpu.c_macros_ref.html#_QUREUl8w"><span class="b">ADDR</span></a>
<a name="385" /><span class="Maybe">     385:</span> 
<a name="386" /><span class="Maybe">     386:</span> <span class="f">#</span><span class="n">ifdef</span> <a href="cpu.c_macros_ref.html#_X19LRVJORUxfX18w"><span class="b">__KERNEL__</span></a>
<a name="387" /><span class="Maybe">     387:</span> <span class="k">/**</span>
<a name="388" /><span class="Maybe">     388:</span> <span class="k"> * ffs - find first set bit in word</span>
<a name="389" /><span class="Maybe">     389:</span> <span class="k"> * @x: the word to search</span>
<a name="390" /><span class="Maybe">     390:</span> <span class="k"> *</span>
<a name="391" /><span class="Maybe">     391:</span> <span class="k"> * This is defined the same way as the libc and compiler builtin ffs</span>
<a name="392" /><span class="Maybe">     392:</span> <span class="k"> * routines, therefore differs in spirit from the other bitops.</span>
<a name="393" /><span class="Maybe">     393:</span> <span class="k"> *</span>
<a name="394" /><span class="Maybe">     394:</span> <span class="k"> * ffs(value) returns 0 if value is 0 or the position of the first</span>
<a name="395" /><span class="Maybe">     395:</span> <span class="k"> * set bit if value is nonzero. The first (least significant) bit</span>
<a name="396" /><span class="Maybe">     396:</span> <span class="k"> * is at position 1.</span>
<a name="397" /><span class="Maybe">     397:</span> <span class="k"> */</span>
<a name="398" /><span class="Maybe">     398:</span> <span class="m">static</span> <span class="m">inline</span> <span class="m">int</span> <span class="b">ffs</span><span class="f">(</span><span class="m">int</span> <span class="b">x</span><span class="f">)</span>
<a name="399" /><span class="Maybe">     399:</span> <span class="f">{</span>
<a name="400" /><span class="Maybe">     400:</span>     <span class="m">int</span> <span class="b">r</span><span class="f">;</span>
<a name="401" /><span class="Maybe">     401:</span> 
<a name="402" /><span class="Maybe">     402:</span> <span class="f">#</span><span class="n">ifdef</span> <a href="cpu.c_macros_ref.html#_Q09ORklHX1g4Nl82NF8w"><span class="b">CONFIG_X86_64</span></a>
<a name="403" /><span class="Maybe">     403:</span>     <span class="k">/*</span>
<a name="404" /><span class="Maybe">     404:</span> <span class="k">     * AMD64 says BSFL won&apos;t clobber the dest reg if x==0; Intel64 says the</span>
<a name="405" /><span class="Maybe">     405:</span> <span class="k">     * dest reg is undefined if x==0, but their CPU architect says its</span>
<a name="406" /><span class="Maybe">     406:</span> <span class="k">     * value is written to set it to the same as before, except that the</span>
<a name="407" /><span class="Maybe">     407:</span> <span class="k">     * top 32 bits will be cleared.</span>
<a name="408" /><span class="Maybe">     408:</span> <span class="k">     *</span>
<a name="409" /><span class="Maybe">     409:</span> <span class="k">     * We cannot do this on 32 bits because at the very least some</span>
<a name="410" /><span class="Maybe">     410:</span> <span class="k">     * 486 CPUs did not behave this way.</span>
<a name="411" /><span class="Maybe">     411:</span> <span class="k">     */</span>
<a name="412" /><span class="Maybe">     412:</span>     <span class="m">asm</span><span class="f">(</span><span class="e">&quot;bsfl %1,%0&quot;</span>
<a name="413" /><span class="Maybe">     413:</span>         <span class="f">:</span> <span class="e">&quot;=r&quot;</span> <span class="f">(</span><span class="b">r</span><span class="f">)</span>
<a name="414" /><span class="Maybe">     414:</span>         <span class="f">:</span> <span class="e">&quot;rm&quot;</span> <span class="f">(</span><span class="b">x</span><span class="f">)</span><span class="f">,</span> <span class="e">&quot;0&quot;</span> <span class="f">(</span><span class="f">-</span><span class="c">1</span><span class="f">)</span><span class="f">)</span><span class="f">;</span>
<a name="415" /><span class="False">     415:</span> <span class="f">#</span><span class="n">elif</span> <span class="b">defined</span><span class="f">(</span><a href="cpu.c_macros_ref.html#_Q09ORklHX1g4Nl9DTU9WXzA_"><span class="b">CONFIG_X86_CMOV</span></a><span class="f">)</span>
<a name="416" /><span class="False">     416:</span>     <span class="m">asm</span><span class="f">(</span><span class="e">&quot;bsfl %1,%0\n\t&quot;</span>
<a name="417" /><span class="False">     417:</span>         <span class="e">&quot;cmovzl %2,%0&quot;</span>
<a name="418" /><span class="False">     418:</span>         <span class="f">:</span> <span class="e">&quot;=&amp;r&quot;</span> <span class="f">(</span><span class="b">r</span><span class="f">)</span> <span class="f">:</span> <span class="e">&quot;rm&quot;</span> <span class="f">(</span><span class="b">x</span><span class="f">)</span><span class="f">,</span> <span class="e">&quot;r&quot;</span> <span class="f">(</span><span class="f">-</span><span class="c">1</span><span class="f">)</span><span class="f">)</span><span class="f">;</span>
<a name="419" /><span class="False">     419:</span> <span class="f">#</span><span class="n">else</span>
<a name="420" /><span class="False">     420:</span>     <span class="m">asm</span><span class="f">(</span><span class="e">&quot;bsfl %1,%0\n\t&quot;</span>
<a name="421" /><span class="False">     421:</span>         <span class="e">&quot;jnz 1f\n\t&quot;</span>
<a name="422" /><span class="False">     422:</span>         <span class="e">&quot;movl $-1,%0\n&quot;</span>
<a name="423" /><span class="False">     423:</span>         <span class="e">&quot;1:&quot;</span> <span class="f">:</span> <span class="e">&quot;=r&quot;</span> <span class="f">(</span><span class="b">r</span><span class="f">)</span> <span class="f">:</span> <span class="e">&quot;rm&quot;</span> <span class="f">(</span><span class="b">x</span><span class="f">)</span><span class="f">)</span><span class="f">;</span>
<a name="424" /><span class="Maybe">     424:</span> <span class="f">#</span><span class="n">endif</span>
<a name="425" /><span class="Maybe">     425:</span>     <span class="m">return</span> <span class="b">r</span> <span class="f">+</span> <span class="c">1</span><span class="f">;</span>
<a name="426" /><span class="Maybe">     426:</span> <span class="f">}</span>
<a name="427" /><span class="Maybe">     427:</span> 
<a name="428" /><span class="Maybe">     428:</span> <span class="k">/**</span>
<a name="429" /><span class="Maybe">     429:</span> <span class="k"> * fls - find last set bit in word</span>
<a name="430" /><span class="Maybe">     430:</span> <span class="k"> * @x: the word to search</span>
<a name="431" /><span class="Maybe">     431:</span> <span class="k"> *</span>
<a name="432" /><span class="Maybe">     432:</span> <span class="k"> * This is defined in a similar way as the libc and compiler builtin</span>
<a name="433" /><span class="Maybe">     433:</span> <span class="k"> * ffs, but returns the position of the most significant set bit.</span>
<a name="434" /><span class="Maybe">     434:</span> <span class="k"> *</span>
<a name="435" /><span class="Maybe">     435:</span> <span class="k"> * fls(value) returns 0 if value is 0 or the position of the last</span>
<a name="436" /><span class="Maybe">     436:</span> <span class="k"> * set bit if value is nonzero. The last (most significant) bit is</span>
<a name="437" /><span class="Maybe">     437:</span> <span class="k"> * at position 32.</span>
<a name="438" /><span class="Maybe">     438:</span> <span class="k"> */</span>
<a name="439" /><span class="Maybe">     439:</span> <span class="m">static</span> <span class="m">inline</span> <span class="m">int</span> <span class="b">fls</span><span class="f">(</span><span class="m">int</span> <span class="b">x</span><span class="f">)</span>
<a name="440" /><span class="Maybe">     440:</span> <span class="f">{</span>
<a name="441" /><span class="Maybe">     441:</span>     <span class="m">int</span> <span class="b">r</span><span class="f">;</span>
<a name="442" /><span class="Maybe">     442:</span> 
<a name="443" /><span class="Maybe">     443:</span> <span class="f">#</span><span class="n">ifdef</span> <a href="cpu.c_macros_ref.html#_Q09ORklHX1g4Nl82NF8w"><span class="b">CONFIG_X86_64</span></a>
<a name="444" /><span class="Maybe">     444:</span>     <span class="k">/*</span>
<a name="445" /><span class="Maybe">     445:</span> <span class="k">     * AMD64 says BSRL won&apos;t clobber the dest reg if x==0; Intel64 says the</span>
<a name="446" /><span class="Maybe">     446:</span> <span class="k">     * dest reg is undefined if x==0, but their CPU architect says its</span>
<a name="447" /><span class="Maybe">     447:</span> <span class="k">     * value is written to set it to the same as before, except that the</span>
<a name="448" /><span class="Maybe">     448:</span> <span class="k">     * top 32 bits will be cleared.</span>
<a name="449" /><span class="Maybe">     449:</span> <span class="k">     *</span>
<a name="450" /><span class="Maybe">     450:</span> <span class="k">     * We cannot do this on 32 bits because at the very least some</span>
<a name="451" /><span class="Maybe">     451:</span> <span class="k">     * 486 CPUs did not behave this way.</span>
<a name="452" /><span class="Maybe">     452:</span> <span class="k">     */</span>
<a name="453" /><span class="Maybe">     453:</span>     <span class="m">asm</span><span class="f">(</span><span class="e">&quot;bsrl %1,%0&quot;</span>
<a name="454" /><span class="Maybe">     454:</span>         <span class="f">:</span> <span class="e">&quot;=r&quot;</span> <span class="f">(</span><span class="b">r</span><span class="f">)</span>
<a name="455" /><span class="Maybe">     455:</span>         <span class="f">:</span> <span class="e">&quot;rm&quot;</span> <span class="f">(</span><span class="b">x</span><span class="f">)</span><span class="f">,</span> <span class="e">&quot;0&quot;</span> <span class="f">(</span><span class="f">-</span><span class="c">1</span><span class="f">)</span><span class="f">)</span><span class="f">;</span>
<a name="456" /><span class="False">     456:</span> <span class="f">#</span><span class="n">elif</span> <span class="b">defined</span><span class="f">(</span><a href="cpu.c_macros_ref.html#_Q09ORklHX1g4Nl9DTU9WXzA_"><span class="b">CONFIG_X86_CMOV</span></a><span class="f">)</span>
<a name="457" /><span class="False">     457:</span>     <span class="m">asm</span><span class="f">(</span><span class="e">&quot;bsrl %1,%0\n\t&quot;</span>
<a name="458" /><span class="False">     458:</span>         <span class="e">&quot;cmovzl %2,%0&quot;</span>
<a name="459" /><span class="False">     459:</span>         <span class="f">:</span> <span class="e">&quot;=&amp;r&quot;</span> <span class="f">(</span><span class="b">r</span><span class="f">)</span> <span class="f">:</span> <span class="e">&quot;rm&quot;</span> <span class="f">(</span><span class="b">x</span><span class="f">)</span><span class="f">,</span> <span class="e">&quot;rm&quot;</span> <span class="f">(</span><span class="f">-</span><span class="c">1</span><span class="f">)</span><span class="f">)</span><span class="f">;</span>
<a name="460" /><span class="False">     460:</span> <span class="f">#</span><span class="n">else</span>
<a name="461" /><span class="False">     461:</span>     <span class="m">asm</span><span class="f">(</span><span class="e">&quot;bsrl %1,%0\n\t&quot;</span>
<a name="462" /><span class="False">     462:</span>         <span class="e">&quot;jnz 1f\n\t&quot;</span>
<a name="463" /><span class="False">     463:</span>         <span class="e">&quot;movl $-1,%0\n&quot;</span>
<a name="464" /><span class="False">     464:</span>         <span class="e">&quot;1:&quot;</span> <span class="f">:</span> <span class="e">&quot;=r&quot;</span> <span class="f">(</span><span class="b">r</span><span class="f">)</span> <span class="f">:</span> <span class="e">&quot;rm&quot;</span> <span class="f">(</span><span class="b">x</span><span class="f">)</span><span class="f">)</span><span class="f">;</span>
<a name="465" /><span class="Maybe">     465:</span> <span class="f">#</span><span class="n">endif</span>
<a name="466" /><span class="Maybe">     466:</span>     <span class="m">return</span> <span class="b">r</span> <span class="f">+</span> <span class="c">1</span><span class="f">;</span>
<a name="467" /><span class="Maybe">     467:</span> <span class="f">}</span>
<a name="468" /><span class="Maybe">     468:</span> 
<a name="469" /><span class="Maybe">     469:</span> <span class="k">/**</span>
<a name="470" /><span class="Maybe">     470:</span> <span class="k"> * fls64 - find last set bit in a 64-bit word</span>
<a name="471" /><span class="Maybe">     471:</span> <span class="k"> * @x: the word to search</span>
<a name="472" /><span class="Maybe">     472:</span> <span class="k"> *</span>
<a name="473" /><span class="Maybe">     473:</span> <span class="k"> * This is defined in a similar way as the libc and compiler builtin</span>
<a name="474" /><span class="Maybe">     474:</span> <span class="k"> * ffsll, but returns the position of the most significant set bit.</span>
<a name="475" /><span class="Maybe">     475:</span> <span class="k"> *</span>
<a name="476" /><span class="Maybe">     476:</span> <span class="k"> * fls64(value) returns 0 if value is 0 or the position of the last</span>
<a name="477" /><span class="Maybe">     477:</span> <span class="k"> * set bit if value is nonzero. The last (most significant) bit is</span>
<a name="478" /><span class="Maybe">     478:</span> <span class="k"> * at position 64.</span>
<a name="479" /><span class="Maybe">     479:</span> <span class="k"> */</span>
<a name="480" /><span class="Maybe">     480:</span> <span class="f">#</span><span class="n">ifdef</span> <a href="cpu.c_macros_ref.html#_Q09ORklHX1g4Nl82NF8w"><span class="b">CONFIG_X86_64</span></a>
<a name="481" /><span class="Maybe">     481:</span> <span class="m">static</span> <a href="cpu.c_macros_ref.html#_X19hbHdheXNfaW5saW5lXzA_"><span class="b">__always_inline</span></a> <span class="m">int</span> <span class="b">fls64</span><span class="f">(</span><span class="b">__u64</span> <span class="b">x</span><span class="f">)</span>
<a name="482" /><span class="Maybe">     482:</span> <span class="f">{</span>
<a name="483" /><span class="Maybe">     483:</span>     <span class="m">int</span> <span class="b">bitpos</span> <span class="f">=</span> <span class="f">-</span><span class="c">1</span><span class="f">;</span>
<a name="484" /><span class="Maybe">     484:</span>     <span class="k">/*</span>
<a name="485" /><span class="Maybe">     485:</span> <span class="k">     * AMD64 says BSRQ won&apos;t clobber the dest reg if x==0; Intel64 says the</span>
<a name="486" /><span class="Maybe">     486:</span> <span class="k">     * dest reg is undefined if x==0, but their CPU architect says its</span>
<a name="487" /><span class="Maybe">     487:</span> <span class="k">     * value is written to set it to the same as before.</span>
<a name="488" /><span class="Maybe">     488:</span> <span class="k">     */</span>
<a name="489" /><span class="Maybe">     489:</span>     <span class="m">asm</span><span class="f">(</span><span class="e">&quot;bsrq %1,%q0&quot;</span>
<a name="490" /><span class="Maybe">     490:</span>         <span class="f">:</span> <span class="e">&quot;+r&quot;</span> <span class="f">(</span><span class="b">bitpos</span><span class="f">)</span>
<a name="491" /><span class="Maybe">     491:</span>         <span class="f">:</span> <span class="e">&quot;rm&quot;</span> <span class="f">(</span><span class="b">x</span><span class="f">)</span><span class="f">)</span><span class="f">;</span>
<a name="492" /><span class="Maybe">     492:</span>     <span class="m">return</span> <span class="b">bitpos</span> <span class="f">+</span> <span class="c">1</span><span class="f">;</span>
<a name="493" /><span class="Maybe">     493:</span> <span class="f">}</span>
<a name="494" /><span class="False">     494:</span> <span class="f">#</span><span class="n">else</span>
<a name="495" /><span class="False">     495:</span> <span class="f">#</span><span class="n">include</span> <span class="f">&lt;</span><span class="m">asm</span><span class="f">-</span><span class="b">generic</span><span class="f">/</span><span class="b">bitops</span><span class="f">/</span><span class="b">fls64</span><span class="f">.</span><span class="b">h</span><span class="f">&gt;</span>
<a name="496" /><span class="Maybe">     496:</span> <span class="f">#</span><span class="n">endif</span>
<a name="497" /><span class="Maybe">     497:</span> 
<a name="498" /><span class="Maybe">     498:</span> <span class="f">#</span><span class="n">include</span> <span class="f">&lt;</span><span class="m">asm</span><span class="f">-</span><span class="b">generic</span><span class="f">/</span><span class="b">bitops</span><span class="f">/</span><span class="b">find</span><span class="f">.</span><span class="b">h</span><span class="f">&gt;</span>
<a name="499" /><span class="Maybe">     499:</span> 
<a name="500" /><span class="Maybe">     500:</span> <span class="f">#</span><span class="n">include</span> <span class="f">&lt;</span><span class="m">asm</span><span class="f">-</span><span class="b">generic</span><span class="f">/</span><span class="b">bitops</span><span class="f">/</span><span class="b">sched</span><span class="f">.</span><span class="b">h</span><span class="f">&gt;</span>
<a name="501" /><span class="Maybe">     501:</span> 
<a name="502" /><span class="Maybe">     502:</span> <span class="f">#</span><span class="n">define</span> <a href="cpu.c_macros_noref.html#_QVJDSF9IQVNfRkFTVF9NVUxUSVBMSUVSXzA_"><span class="b">ARCH_HAS_FAST_MULTIPLIER</span></a> <span class="c">1</span>
<a name="503" /><span class="Maybe">     503:</span> 
<a name="504" /><span class="Maybe">     504:</span> <span class="f">#</span><span class="n">include</span> <span class="f">&lt;</span><span class="m">asm</span><span class="f">/</span><span class="b">arch_hweight</span><span class="f">.</span><span class="b">h</span><span class="f">&gt;</span>
<a name="505" /><span class="Maybe">     505:</span> 
<a name="506" /><span class="Maybe">     506:</span> <span class="f">#</span><span class="n">include</span> <span class="f">&lt;</span><span class="m">asm</span><span class="f">-</span><span class="b">generic</span><span class="f">/</span><span class="b">bitops</span><span class="f">/</span><span class="b">const_hweight</span><span class="f">.</span><span class="b">h</span><span class="f">&gt;</span>
<a name="507" /><span class="Maybe">     507:</span> 
<a name="508" /><span class="Maybe">     508:</span> <span class="f">#</span><span class="n">include</span> <span class="f">&lt;</span><span class="m">asm</span><span class="f">-</span><span class="b">generic</span><span class="f">/</span><span class="b">bitops</span><span class="f">/</span><span class="b">le</span><span class="f">.</span><span class="b">h</span><span class="f">&gt;</span>
<a name="509" /><span class="Maybe">     509:</span> 
<a name="510" /><span class="Maybe">     510:</span> <span class="f">#</span><span class="n">include</span> <span class="f">&lt;</span><span class="m">asm</span><span class="f">-</span><span class="b">generic</span><span class="f">/</span><span class="b">bitops</span><span class="f">/</span><span class="b">ext2</span><span class="f">-</span><span class="b">atomic</span><span class="f">-</span><span class="b">setbit</span><span class="f">.</span><span class="b">h</span><span class="f">&gt;</span>
<a name="511" /><span class="Maybe">     511:</span> 
<a name="512" /><span class="Maybe">     512:</span> <span class="f">#</span><span class="n">endif</span> <span class="k">/* __KERNEL__ */</span>
<a name="513" /><span class="True">     513:</span> <span class="f">#</span><span class="n">endif</span> <span class="k">/* _ASM_X86_BITOPS_H */</span>
<a name="514" /><span class="True">     514:</span> </pre>
  </body>
</html>
